{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lasagne.layers import *",
    "\n",
    "from lasagne.nonlinearities import *",
    "\n",
    "from lasagne import init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn = InputLayer([None, 3, 100, 100])",
    "\n",
    "\n",
    "nn = Conv2DLayer(nn, num_filters=512, filter_size=(3, 3),",
    "\n",
    "                 W=init.Constant(0))",
    "\n",
    "\n",
    "nn = Conv2DLayer(nn, num_filters=128, filter_size=(3, 3),",
    "\n",
    "                 W=init.Constant(0))",
    "\n",
    "\n",
    "nn = Conv2DLayer(nn, num_filters=32, filter_size=(3, 3),",
    "\n",
    "                 W=init.Constant(0))",
    "\n",
    "\n",
    "nn = Pool2DLayer(nn, pool_size=(6, 6), mode='max')",
    "\n",
    "\n",
    "nn = Conv2DLayer(nn, num_filters=8, filter_size=(10, 10),",
    "\n",
    "                 W=init.Normal(std=0.01))",
    "\n",
    "\n",
    "nn = Conv2DLayer(nn, num_filters=8, filter_size=(10, 10),",
    "\n",
    "                 W=init.Normal(std=0.01))",
    "\n",
    "\n",
    "nn = Pool2DLayer(nn, pool_size=(3, 3), mode='max')",
    "\n",
    "\n",
    "nn = DenseLayer(nn, 512, nonlinearity=softmax)",
    "\n",
    "\n",
    "nn = DropoutLayer(nn, p=0.5)",
    "\n",
    "\n",
    "nn = DenseLayer(nn, 512, nonlinearity=softmax)",
    "\n",
    "\n",
    "nn = DenseLayer(nn, 10, nonlinearity=sigmoid)",
    "\n",
    "\n",
    "nn = DropoutLayer(nn, p=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "# Book of grudges\n",
    "* zero init for weights will cause symmetry effect\n",
    "* Too many filters for first 3x3 convolution - will lead to enormous matrix while there's just not enough relevant combinations of 3x3 images (overkill).\n",
    "* Usually the further you go, the more filters you need.\n",
    "* large filters (10x10 is generally a bad pactice, and you definitely need more than 10 of them\n",
    "* the second of 10x10 convolution gets 8x6x6 image as input, so it's technically unable to perform such convolution.\n",
    "* Softmax nonlinearity effectively makes only 1 or a few neurons from the entire layer to \"fire\", rendering 512-neuron layer almost useless. Softmax at the output layer is okay though\n",
    "* Dropout after probability prediciton is just lame. A few random classes get probability of 0, so your probabilities no longer sum to 1 and crossentropy goes -inf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
